
#' Return a table of conjoint effects
#'
#' @param conjoint conjoint data
#' @param use a character vector taking on 6 possible values:
#' "agree" returns results conditioning on respondents who agree with each attribute value 
#' "disagree" returns results conditioning on respondents who disagree with each attribute value 
#' "both" returns results coded in the direction of agreement with the respondent's position
#' "direction" returns results conditioning on directions supplied by the direction parameter
#' "overall" returns unconditional effects, but only for the attributes where a position is available
#' "all" returns unconditional effects for all attributes
#' @param grouped_data logical flag for whether a variable "group" exists in the data and should
#' be used to calculate effects by group. by default this is true if the data contains "group"
#' @param weights optional vector of weights
#' @param direction optional data frame coding the direction of each attribute
#' @param lab optional data frame coding readable names of each attribute
#' @param bonferroni logical variable for whether bonferroni correction should be used
#' 
#' @return table of conjoint results
#' @export
#' @importFrom lfe tidyverse
#' @examples 
table_conjoint = function(conjoint,use="both",weights=rep(1,dim(conjoint)[1]),grouped_data=has_name(conjoint,"group"),
                                      direction=NULL,lab=NULL,bonferroni=F){
  print("Preparing Policies")
  # Extract all of the individual things we want an effect for -- the
  # mechanical questions and the categorical questions.
  # Which columns actually have conjoint data to fill in?
  exclude_set = c(
    colnames(conjoint)[str_detect(colnames(conjoint), "demog_")],
    colnames(conjoint)[str_detect(colnames(conjoint), "policy_")]
  )
  if(! use == "all"){exclude_set = c(exclude_set,
                                     colnames(conjoint)[str_detect(colnames(conjoint), "_pres")],
                                     colnames(conjoint)[str_detect(colnames(conjoint), "_local")])}
  
  which_effects_conjoint =
    setdiff(
      colnames(conjoint),
      # Which columns actually have conjoint data to fill in?
      c("choice", "person_id", "question", "set","ResponseId",
        "user_agent","user_agent_type","user_agent_ver","user_agent_os","user_agent_os_ver",
         "week","group","wave","race_pres","race_local", exclude_set)) # "race_pres", "race_local" not sure why these were in here
  
  # Get names of the associated policy questions, and compute agreement scores
  policies <- paste("policy_",which_effects_conjoint,sep="")
  
  # Subset to items where we have the associated policy question, unless use="all"
  if(use != "all"){
    which_effects_conjoint = which_effects_conjoint[policies %in% colnames(conjoint)]
    policies = policies[policies %in% colnames(conjoint)]
  }
    
  # Get correct order for direction vector from "lab"
  if(! is.null(direction)){
    direction <- direction$direction[match(which_effects_conjoint,direction$label)]
  }
  
  # Code agreement with the positions in the conjoint
  for(i in which_effects_conjoint){ if (! use %in% c("overall","all")){
    conjoint[[paste("agreement_",i,sep="")]] <-
      ifelse(conjoint[[i]]==conjoint[[paste("policy_",i,sep="")]],1,0)
    conjoint[[paste("agreement_",i,sep="")]][conjoint[[i]] %in% c(3,999)] <- NA
    conjoint[[paste("agreement_",i,sep="")]][is.na(conjoint[[i]])] <- NA
  }}
  positions_agreement <- paste("agreement_",which_effects_conjoint,sep="")
  
  # overwrite this if we want overall effects
  if(use %in% c("overall","all")){positions_agreement <- which_effects_conjoint}
  
  # attach the weights to the dataset for future use
  conjoint$w <- weights
  
  # Map each column into a conjoint
  print("Computing Results For...")
  result = positions_agreement %>% map(function(effect_name) {
    # Select what we want, rename the column to effect to avoid the
    # messy NSE stuff. What does !!effect_name do? Expands the vector of
    # variable names into arguments for select. So we're selecting choice,
    # person_id, everything in effect_name, and group if the data is grouped
    print(effect_name)
    if(grouped_data){data = conjoint %>% select(choice, person_id, !!effect_name, group) %>%
      # Rename whatever the variable's actual name is to effect so we can proceed more easily
      rename_at(3, ~ "effect")}
    if(! grouped_data){data = conjoint %>% select(choice, person_id, !!effect_name) %>%
      # Rename whatever the variable's actual name is to effect so we can proceed more easily
      rename_at(3, ~ "effect")}
      
    ### Print error if effect is empty
    if(! F %in% is.na(data$effect)){print(paste("Data is empty for ",effect_name,"!",sep=""))}
    
    ### calculate weighted mean support for the policy among group 0
    ### (if the policy is available)
    tmp <- distinct(conjoint,person_id,.keep_all=T)
    pol <- policies[which(positions_agreement==effect_name)]
    if(pol %in% names(tmp)){
      if(grouped_data){margin_yes <- weighted.mean(case_when(tmp[[pol]]==2~0,tmp[[pol]]==1~1,tmp[[pol]]==3~0,
                                                             tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp$group==0 & tmp[[pol]]!=999],
                            w=tmp$w[tmp$group==0 & tmp[[pol]]!=999],na.rm=T)}
      if(! grouped_data){margin_yes <- weighted.mean(case_when(tmp[[pol]]==2~0,tmp[[pol]]==1~1,tmp[[pol]]==3~0,
                                                               tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp[[pol]]!=999],
                                               w=tmp$w[tmp[[pol]]!=999],na.rm=T)}
      if(grouped_data){margin_no <- weighted.mean(case_when(tmp[[pol]]==2~1,tmp[[pol]]==1~0,tmp[[pol]]==3~0,
                                                            tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp$group==0 & tmp[[pol]]!=999],
                                               w=tmp$w[tmp$group==0 & tmp[[pol]]!=999],na.rm=T)}
      if(! grouped_data){margin_no <- weighted.mean(case_when(tmp[[pol]]==2~1,tmp[[pol]]==1~0,tmp[[pol]]==3~0,
                                                              tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp[[pol]]!=999],
                                                 w=tmp$w[tmp[[pol]]!=999],na.rm=T)}
    }
    if(! pol %in% names(tmp)){ margin_yes <- NA 
    margin_no <- NA}
    
    # condition on respondents in favor of the policy
    w <- weights
    if(use=="for"){
      pol <- policies[which(positions_agreement==effect_name)]
      data <- data[which(conjoint[[pol]]==1),]
      w <- w[which(conjoint[[pol]]==1)]
    }
    # condition on respondents against the policy
    if(use=="against"){
      pol <- policies[which(positions_agreement==effect_name)]
      data <- data[which(conjoint[[pol]]==2),]
      w <- w[which(conjoint[[pol]]==2)]
    }
    # condition on respondents DK or didn't answer on the the policy
    if(use=="DK"){
      pol <- policies[which(positions_agreement==effect_name)]
      data <- data[which(is.na(conjoint[[pol]]) | conjoint[[pol]]==3),]
      w <- w[which(is.na(conjoint[[pol]]))]
    }
    # condition on respondents taking a particular stance on each policy
    if(use=="direction"){
      pol <- policies[which(positions_agreement==effect_name)]
      dir <- direction[which(positions_agreement==effect_name)]
      data <- data[which(conjoint[[pol]]==dir),]
      w <- w[which(conjoint[[pol]]==dir)]
    }
    
    if(use %in% c("overall","all")){
      data$effect <- abs(data$effect - 2)
    }
    
    # FELM: Formula: regression (outcome on treatment). Second group is
    # fixed effects. Third group is instrumental variables. Fourth group
    # is what to cluster on.
    group_error=F
    if(grouped_data){if(0 %in% rowSums(table(data$group,data$effect))){
      print(paste("Group perfectly predicts treatment assignment for:",effect_name))
      group_error=T
      grouped_data=F
    }}
    if(! grouped_data){temp_cl = felm(choice ~ effect | 0 | 0 | person_id, data = data,weights=w)}
    if(grouped_data){
      # these two regressions are basically the same:
      data$group <- as.character(data$group)
      # first regression gets us significant differences from the effect in group 0
      temp_cl = felm(choice ~ effect + effect*group | 0 | 0 | person_id, data = data,weights=w)
      # second regression gets us main effects with correct SEs for each group
      temp_cl2 = felm(choice ~ effect:group + group | 0 | 0 | person_id, data = data,weights=w)
    }
    
    t_value <- 1.96
    if(bonferroni){t_value <- qnorm(1-.025/length(positions_agreement))}

    # Return the results with a CI and significance
    output <- tibble(
      condition = str_remove(effect_name,"agreement_"),
      effect = temp_cl$coefficients[2],
      se = temp_cl$cse[2],
      ci_low = effect - t_value * se,
      ci_high = effect + t_value*se,
      is_sig =
        case_when(
          (ci_high < 0) | (ci_low > 0) ~ 1,
          (ci_high > 0) & (ci_low < 0) ~ 0
        ),
      ci_int_low = ifelse(grouped_data,
                          temp_cl$coefficients[4] - t_value*temp_cl$cse[4],NA),
      ci_int_high = ifelse(grouped_data,
                           temp_cl$coefficients[4] + t_value*temp_cl$cse[4],NA),
      is_sig_int =
        case_when(
          (ci_int_high < 0) | (ci_int_low > 0) ~ 1,
          (ci_int_high > 0) & (ci_int_low < 0) ~ 0
        ),
      N_obs = ifelse(grouped_data,
                     length(which(c(data$group==0)[-temp_cl$na.action])),
                     temp_cl$N),
      N_people= ifelse(grouped_data,
                       length(unique(data$person_id[data$group==0 & ! 1:dim(data)[1] %in% temp_cl$na.action])),
                       length(unique(data$person_id[! 1:dim(data)[1] %in% temp_cl$na.action]))),
      margin_yes=margin_yes,
      margin_no=margin_no,
      group=0
    )
    if(group_error){output$effect = NA}

    if(grouped_data){
    fulloutput <- output
    ngroups <- length(unique(data$group[! is.na(data$group)]))
    for(i in 2:ngroups){
      tmpoutput <- tibble(
        condition = str_remove(effect_name,"agreement_"),
        effect = temp_cl2$coefficients[ngroups + i],
        se = temp_cl2$cse[ngroups + i],
        ci_low = effect - t_value * se,
        ci_high = effect + t_value*se,
        is_sig =
          case_when(
            (ci_high < 0) | (ci_low > 0) ~ 1,
            (ci_high > 0) & (ci_low < 0) ~ 0
          ),
        ci_int_low = temp_cl$coefficients[ngroups + i] - t_value*temp_cl$cse[ngroups + i],
        ci_int_high = temp_cl$coefficients[ngroups + i] + t_value*temp_cl$cse[ngroups + i],
        is_sig_int =
          case_when(
            (ci_int_high < 0) | (ci_int_low > 0) ~ 1,
            (ci_int_high > 0) & (ci_int_low < 0) ~ 0
          ),
        N_obs = length(which(c(data$group==i-1)[-temp_cl$na.action])),
        N_people= length(unique(data$person_id[data$group==i-1 & ! 1:dim(data)[1] %in% temp_cl$na.action])),
        margin_yes = ifelse(pol %in% names(tmp),weighted.mean(case_when(tmp[[pol]]==2~0,tmp[[pol]]==1~1,tmp[[pol]]==3~0,
                                             tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp$group==(i-1) & tmp[[pol]]!=999],
                                   w=tmp$w[tmp$group==(i-1) & tmp[[pol]]!=999],na.rm=T), NA),
#            ifelse(pol %in% names(tmp),
#                        weighted.mean(case_when(tmp[[pol]]==2~0,tmp[[pol]]==1~1,is.na(tmp[[pol]])~0)[tmp$group==(i-1)],
#                                      w=tmp$w[tmp$group==(i-1)],na.rm=T), NA),
        margin_no = ifelse(pol %in% names(tmp),weighted.mean(case_when(tmp[[pol]]==2~1,tmp[[pol]]==1~0,tmp[[pol]]==3~0,
                                            tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp$group==(i-1) & tmp[[pol]]!=999],
                                  w=tmp$w[tmp$group==(i-1) & tmp[[pol]]!=999],na.rm=T),NA),
#  ifelse(pol %in% names(tmp),
#                            weighted.mean(case_when(tmp[[pol]]==2~1,tmp[[pol]]==1~0,is.na(tmp[[pol]])~0)[tmp$group==(i-1)],
#                                          w=tmp$w[tmp$group==(i-1)],na.rm=T), NA),
        group=i-1
      )
      fulloutput <- bind_rows(fulloutput,tmpoutput)
    }
    output <- fulloutput
    }
    
    if(use=="direction"){
      dir <- direction[which(positions_agreement==effect_name)]
      # reverse the margin when the direction is reversed
      #if(dir==2){output$margin <- 1-output$margin}
    }
    if(use=="against"){
      # reverse the margin
      #output$margin <- 1-output$margin
    }
    
    output
  }) %>%
    # bind_rows will combine across all the coefficients.
    bind_rows() %>%
    # Sort data internally
    arrange(-effect) %>%
    # Reorder by effect size descending as far as ggplot2 is concerned
    mutate(condition = fct_reorder(condition, effect))
  
  # match on descriptive labels, if available
  if(! is.null(lab)){
    result$condition <- as.character(result$condition)
    ind <- match(result$condition,lab$label)
    result$label <- lab$newlabel[ind]
    # if the direction is provided, add this information to the labels as well
    if(use=="direction"){
      ind <- match(result$condition,which_effects_conjoint)
      result$label <- paste(c("","(not) ")[direction[ind]],result$label,sep="")
    }
    result$condition <- factor(result$condition)
  }
  
  # Return table 
  result
}


plot_conjoint = function(conjoint,use="both",bonferroni=F,grouped_data=has_name(conjoint,"group"),
                         color_set=c("1"="grey","2"="black","3"="lightblue","4"="blue"),
                         lab=NULL,order="set1",shape_set=c(1,2,3),direction=NULL,title="",
                                weights=rep(1,dim(conjoint)[1]),
                         is_result = F){
  # Check if conjoint is result of table_conjoint() already
  if(!is_result){
    result <- table_conjoint(conjoint=conjoint,use=use,bonferroni=bonferroni,
                             lab=lab,direction=direction,grouped_data=grouped_data,
                             weights=weights)
  } else {
    result <- conjoint
  }
  
  
  if(! grouped_data){
    result$group = 0
    result$is_sig_int = result$is_sig
  }
  
  # overwrite margin if use=all so that
  # questions that were not asked have a point size
  if(use=="all"){
    result$margin=.5
  }
  
  # Here's the plot
  shapes = case_when(
    result$group == 0 ~ shape_set[1],
    result$group == 1 ~ shape_set[2],
    result$group == 2 ~ shape_set[3])
  
  dodge <- position_dodge(width=.5)
  
  result$ordering <- -result$effect
  if(order=="diff"){result$ordering <- -result$diff}
  
  # use a descriptive label if available
  result$y = result$condition
  if(has_name(result,"label")){
    result$y = result$label
  }

  the_plot <- ggplot(
    result,
    aes(
      xmin = ci_low,
      xmax = ci_high,
      x = effect,
      y = fct_reorder2(y,-group,ordering),
      shape = factor(shapes),
      color = factor(2*group + is_sig_int + 1)
    )) +
    scale_color_manual(values = color_set) +
    scale_size_identity() +
    geom_point(position=ggstance::position_dodgev(height=0.5), aes(size=(2.5))) +
    geom_errorbarh(position=ggstance::position_dodgev(height=0.5),height=0) +
    geom_vline(aes(xintercept = 0), linetype = "dashed") +
    theme_bw() + 
    theme(legend.position = "none") +
    ggtitle(title) + 
    xlab("") +
    ylab("")
  the_plot
}

plot_conjoint_table = function(result,scale_points=F,grouped_data=FALSE,
                         color_set=c("1"="grey","2"="black","3"="lightblue","4"="blue"),
                         lab=NULL,order="set1",shape_set=c(1,2,3),direction=NULL,title=""){
  if(! grouped_data){
    result$group = 0
    result$is_sig_int = result$is_sig
  }
  
  # Here's the plot
  shapes = case_when(
    result$group == 0 ~ shape_set[1],
    result$group == 1 ~ shape_set[2],
    result$group == 2 ~ shape_set[3])
  
  dodge <- position_dodge(width=.5)
  
  result$ordering <- -result$effect
  if(order=="diff"){result$ordering <- -result$diff}
  
  # use a descriptive label if available
  result$y = result$condition
  if(has_name(result,"label")){
    result$y = result$label
  }
  
  the_plot <- ggplot(
    result,
    aes(
      xmin = ci_low,
      xmax = ci_high, 
      x = effect,
      y = fct_reorder2(y,-group,ordering),
      shape = factor(shapes),
      color = factor(2*group + is_sig_int + 1)
    )) +
    scale_color_manual(values = color_set) +
    scale_size_identity() +
    geom_point(position=ggstance::position_dodgev(height=0.5),
               aes(size=(1.5))) +#5*margin))) + 
    geom_errorbarh(position=ggstance::position_dodgev(height=0.5),height=0) +
    geom_vline(aes(xintercept = 0), linetype = "dashed") +
    theme_bw() + 
    theme(legend.position = "none") +
    ggtitle(title) + 
    xlab("") +
    ylab("")
  the_plot
}


#' Subset conjoint data by dropping all questions that contained attributes
#' outside the given set.
#'
#' @param conjoint conjoint data
#' @param policy_ind index in the data of all of the conjoint attributes
#' @param subset_ind index of the attributes that should be kept
#' 
#' @return subsetted data
#' @export
#' @importFrom
#' @examples 
subset_conjoint <- function(data,policy_ind,subset_ind){
  choices_to_remove <- setdiff(policy_ind,subset_ind)
  not_all_na <- ! is.na(rowMeans(data[,choices_to_remove],na.rm=T))
  person_question <- paste(data$person_id,data$question,sep="-")
  to_remove <- person_question[not_all_na]
  data[! person_question %in% to_remove, -choices_to_remove]
}


#' Run conjoint analysis for each question conditioning on positions
#'
#' @param dat conjoint data
#' @param policies names of policies
#' @param policy_mat data frame including policy names and direction
#' 
#' @return data frame of estimates
#' @export
#' @importFrom
#' @examples 
conjoint_by_question <- function(dat,policies,policy_mat){
  tab_full <- cbind()
  for(i in policies){
    print(i)
    tmp <- dat[! is.na(dat[[i]]),]
    pos <- tmp[[paste("policy",i,sep="_")]]
    tmp$group <- NA
    tmp$group[pos==policy_mat$conservative[policy_mat$label == i]]<- 0
    tmp$group[pos==policy_mat$liberal[policy_mat$label == i]] <- 1
    nacol <- function(x){! FALSE %in% is.na(x)}
    nacols <- apply(tmp,2,nacol)
    if(TRUE %in% nacols){tmp <- tmp[,-which(nacols)]}
    tab <- table_conjoint(conjoint=tmp,lab=policy_mat,
                          weights=tmp$demog_weight,use="both",
                          grouped_data=TRUE)
    tab <- tab[tab$condition==i,]
    tab_full <- rbind(tab_full,tab)
  }
  tab_full
}

###
### function to fix margin labels
###

fix_direction_v2 <- function(mat,policy_mat,policies,direction,dir_var="conservative"){
  for(i in policies){
    if(policy_mat[[dir_var]][policy_mat$label == i]!=direction){
      mat$label[mat$condition==i] = paste("(do not)",mat$label[mat$condition==i])
      ind0 <- mat$condition==i & mat$group==0
      ind1 <- mat$condition==i & mat$group==1
      mat$group[ind0] <- 1
      mat$group[ind1] <- 0
      mat$margin[ind1] <- mat$margin_no[ind1]
      mat$margin[ind0] <- mat$margin_yes[ind0]
    } else {
      ind0 <- mat$condition==i & mat$group==0
      mat$margin[ind0] <- mat$margin_yes[ind0]
      ind1 <- mat$condition==i & mat$group==1
      mat$margin[ind1] <- mat$margin_no[ind1]
    }
  }
  na_ind <- is.na(mat$margin)
  mat$margin <- paste(round(mat$margin*100),"%",sep="")
  mat$margin[na_ind] <- ""
  mat
}


fix_direction <- function(mat,policy_mat,policies,direction){
  for(i in policies){
    if(policy_mat$conservative[policy_mat$label == i]==direction){
      mat$label[mat$condition==i] = paste("(do not)",mat$label[mat$condition==i])
      ind0 <- mat$condition==i & mat$group==0
      ind1 <- mat$condition==i & mat$group==1
      mat$group[ind0] <- 1
      mat$group[ind1] <- 0
      mat$margin[ind1] <- mat$margin_yes[ind1]
      mat$margin[ind0] <- mat$margin_no[ind0]
    } else {
      ind0 <- mat$condition==i & mat$group==0
      mat$margin[ind0] <- mat$margin_no[ind0]    
      ind1 <- mat$condition==i & mat$group==1
      mat$margin[ind1] <- mat$margin_yes[ind1]   
    }
  }
  na_ind <- is.na(mat$margin)
  mat$margin <- paste(round(mat$margin*100),"%",sep="")
  mat$margin[na_ind] <- ""
  mat
}

###
### functions to fix margin labels
###

fix_direction_labels_only <- function(mat,policy_mat,policies,direction){
  for(i in policies){
    if(policy_mat$conservative[policy_mat$label == i]==direction){
      mat$label[mat$condition==i] = paste("(do not)",mat$label[mat$condition==i])
    }
  }
  mat
}

fix_label <- function(mat){
  mat$label[mat$label=="(do not) cash payments (coronavirus)"] <- "(no) cash payments (coronavirus)"
  mat$label[mat$label=="(do not) affirmative action in higher ed"] <- "(no) affirmative action in higher ed"
  mat$label[mat$label=="(do not) extra pandemic unemployment"] <- "(no) extra pandemic unemployment"
  mat$label[mat$label=="(do not) dreamer path to citizenship"] <- "(no) dreamer path to citizenship"
  mat$label[mat$label=="(do not) universal gun background checks"] <- "(no) universal gun background checks"
  mat$label[mat$label=="(do not) $15 minimum wage"] <- "(no) $15 minimum wage"
  mat$label[mat$label=="(do not) broad path to citizenship"] <- "(no) broad path to citizenship"
  mat$label[mat$label=="(do not) mandatory covid vaccine"] <- "(no) mandatory covid vaccine"
  mat$label[mat$label=="(do not) mass deportation"] <- "(no) mass deportation"
  mat$label[mat$label=="(do not) additional police funding"] <- "(no) additional police funding"
  mat$label[mat$label=="(do not) market prices for pre-existing conditions"] <- "(no) market prices for pre-existing conditions"
  mat$label[mat$label=="(do not) abortion any time"] <- "(no) unrestricted abortion"
  mat$label[mat$label=="(do not) debt free state college"] <- "(no) debt free state college"
  mat$label[mat$label=="(do not) dreamer path to citizenship"] <- "(no) dreamer path to citizenship"
  mat$label[mat$label=="(do not) big environmental program"] <- "(no) big environmental program"
  mat$label[mat$label=="(do not) universal jobs guarantee"] <- "(no) universal jobs guarantee"
  mat$label[mat$label=="(do not) federal health insurance"] <- "(no) federal health insurance"
  mat$label[mat$label=="(do not) public guns registry"] <- "(no) public guns registry"
  mat$label[mat$label=="(do not) universal gun background checks"] <- "(no) universal gun background checks"
  mat$label[mat$label=="(do not) low income health subsidy"] <- "(no) low income health subsidy"
  mat$label[mat$label=="(do not) 12 weeks maternity leave"] <- "(no) 12 weeks maternity leave"
  mat$label[mat$label=="(do not) medicare for all"] <- "(no) medicare for all"
  mat$label[mat$label=="(do not) $15 minimum wage"] <- "(no) $15 minimum wage"
  mat$label[mat$label=="(do not) broad path to citizenship"] <- "(no) broad path to citizenship"
  mat$label[mat$label=="(do not) healtchare public option"] <- "(no) healthcare public option"
  mat$label[mat$label=="(do not) slave reparations"] <- "(no) slave reparations"
  mat$label[mat$label=="(do not) no military support for Israel"] <- "(do not cut) military support for Israel"
  mat$label[mat$label=="(do not) abortion coverage optional"] <- "(no) abortion coverage optional"
  mat$label[mat$label=="(do not) mass deportation"] <- "(no) mass deportation"
  mat$label[mat$label=="(do not) merit-based immigration"] <- "(no) merit-based immigration"
  mat$label[mat$label=="(do not) citizenship for wire transfers"] <- "(no) citizenship for wire transfers"
  mat$label[mat$label=="(do not) more oil gas drilling"] <- "(not) more oil gas drilling"
  mat$label[mat$label=="(do not) right to work law"] <- "(no) right to work law"
  mat$label[mat$label=="(do not) ten command in public buildings"] <- "(no) ten command in public buildings"
  mat$label[mat$label=="healthcare public option"] <- "healthcare public option"
  mat$label[mat$label=="(do not) abortion any time"] <- "(no) late abortion"
  mat$label[mat$label=="(do not) cash payments (coronavirus)"] <- "(no) cash payments (coronavirus)"
  mat$label[mat$label=="(do not) dreamer path to citizenship"] <- "(no) dreamer path to citizenship"
  mat$label[mat$label=="(do not) no military support for egypt"] <- "(do not) cut military support for egypt"
  mat$label[mat$label=="(do not) non-partisan districting"] <- "(no) non-partisan districting"
  mat$label[mat$label=="(do not) federal health insurance"] <- "(no) federal health insurance"
  mat$label[mat$label=="(do not) universal gun background checks"] <- "(no) universal gun background checks"
  mat$label[mat$label=="(do not) no military support for Israel"] <- "(do not) cut military support for Israel"
  mat$label[mat$label=="(do not) broad path to citizenship"] <- "(no) broad path to citizenship"
  mat$label[mat$label=="(do not) public election financing"] <- "(no) public election financing"
  mat$label[mat$label=="(do not) no military support for saudi arabia"] <- "(do not) cut military support for saudi arabia"
  mat$label[mat$label=="(do not) wealth tax"] <- "(no) wealth tax"
  mat$label[mat$label=="(do not) tariffs on china"] <- "(no) tariffs on china"
  mat$label[mat$label=="(do not) mass deportation"] <- "(no) mass deportation"
  mat$label[mat$label=="(do not) green new deal"] <- "(no) green new deal"
  mat
}

amce_scatterplot = \(data, demographic, color, lim_lo, lim_hi, agree_in = NA){
  # Correct subtitle/data
  if(!is.na(agree_in)){
    sub_in = paste0(agree_in, " with policy: Correlation = ")
    data = data |> filter(agree == agree_in)
    line_alpha = 0
  } else {
    sub_in = paste0(demographic, ": Correlation = ")
    data = data |> filter(demo==demographic)
    line_alpha = 0.25
  }
  # NOTE: You MUST use invert_group() on the data first, such that:
  # group == 1 = low income/edu and group == 0 = high income/edu
  
  dat = left_join(data %>% filter(group==1),
                  data %>% filter(group==0),
                  by="condition") |> 
    # Create segment data for diagonal CIs
    mutate(halfrange = (ci_int_high.x - ci_int_low.x)/2,
           a = sqrt((halfrange^2)/2),
           x.diag.hi = effect.x - a,
           y.diag.hi = effect.y + a,
           x.diag.lo = effect.x + a,
           y.diag.lo = effect.y - a,
           N_obs = N_obs.x + N_obs.y
    )
  
  dem_out = deming::deming(effect.y ~ effect.x, data = dat,
                           weights = N_obs, xstd = se.x, ystd = se.y)

  
  fig = ggplot(dat, aes(x = effect.x, y = effect.y)) +
    geom_point() +
    geom_segment(aes(x = x.diag.hi, y = y.diag.hi, xend = x.diag.lo, yend = y.diag.lo)) +
    geom_abline(slope = 1, intercept = 0, lty = 2, alpha = .25) +
    #geom_smooth(method = "lm", color = color, fullrange = F) +
    geom_abline(slope = dem_out$coefficient[2], intercept = dem_out$coefficient[1],
                color = color) +
    geom_vline(xintercept = 0, lty = 2, alpha = line_alpha) +
    geom_hline(yintercept = 0, lty = 2, alpha = line_alpha) +
    theme_classic() +
    lims(x = c(lim_lo, lim_hi), y = c(lim_lo, lim_hi)) +
    labs(subtitle = paste0(sub_in,
                           round(with(dat,cor(effect.x,effect.y)), 2)), 
         x = paste0("Low-",demographic," AMCE"),
         y = paste0("High-",demographic," AMCE"))
  return(fig)
}

# group inversion (for plotting references)
invert_group = \(x, demo){
  x |> mutate(group = abs(group - 1))
}
# fix naming error
fix_healthcare = \(x){
  x |> 
    mutate(label = ifelse(label == "healtchare public option", "healthcare public option", label))
}

# Printing convenience function
ccat = \(x){
  cat("\n\n",
      x,
      "\n")
}

# Table convenience function

fix_table = \(x){
  x |> 
    str_remove_all("textbackslash\\{\\}") |> 
    str_replace_all("\\\\\\{","{") |>
    str_replace_all("\\\\\\}","}") |> 
    str_replace_all("makecell\\[r\\]|makecell\\[l\\]", "makecell[c]")
}

# Special Conjoint Table Function for Monthly Effects

table_conjoint_new = function(conjoint,use="both",weights=rep(1,dim(conjoint)[1]),grouped_data=has_name(conjoint,"group"),
                              direction=NULL,lab=NULL,bonferroni=F){
  #print("Preparing Policies")
  # Extract all of the individual things we want an effect for -- the
  # mechanical questions and the categorical questions.
  # Which columns actually have conjoint data to fill in?
  exclude_set = c(
    colnames(conjoint)[str_detect(colnames(conjoint), "demog_")],
    colnames(conjoint)[str_detect(colnames(conjoint), "policy_")]
  )
  if(! use == "all"){exclude_set = c(exclude_set,
                                     colnames(conjoint)[str_detect(colnames(conjoint), "_pres")],
                                     colnames(conjoint)[str_detect(colnames(conjoint), "_local")])}
  
  which_effects_conjoint =
    setdiff(
      colnames(conjoint),
      # Which columns actually have conjoint data to fill in?
      c("choice", "person_id", "question", "set","ResponseId",
        "user_agent","user_agent_type","user_agent_ver","user_agent_os","user_agent_os_ver",
        "week","group","wave","race_pres","race_local", exclude_set)) # "race_pres", "race_local" not sure why these were in here
  
  # Get names of the associated policy questions, and compute agreement scores
  policies <- paste("policy_",which_effects_conjoint,sep="")
  
  # Subset to items where we have the associated policy question, unless use="all"
  if(use != "all"){
    which_effects_conjoint = which_effects_conjoint[policies %in% colnames(conjoint)]
    policies = policies[policies %in% colnames(conjoint)]
  }
  
  # Get correct order for direction vector from "lab"
  if(! is.null(direction)){
    direction <- direction$direction[match(which_effects_conjoint,direction$label)]
  }
  
  # Code agreement with the positions in the conjoint
  for(i in which_effects_conjoint){ if (! use %in% c("overall","all")){
    conjoint[[paste("agreement_",i,sep="")]] <-
      ifelse(conjoint[[i]]==conjoint[[paste("policy_",i,sep="")]],1,0)
    conjoint[[paste("agreement_",i,sep="")]][conjoint[[i]] %in% c(3,999)] <- NA
    conjoint[[paste("agreement_",i,sep="")]][is.na(conjoint[[i]])] <- NA
  }}
  positions_agreement <- paste("agreement_",which_effects_conjoint,sep="")
  
  # overwrite this if we want overall effects
  if(use %in% c("overall","all")){positions_agreement <- which_effects_conjoint}
  
  # attach the weights to the dataset for future use
  conjoint$w <- weights
  
  # Map each column into a conjoint
  #print("Computing Results For...")
  result = positions_agreement %>% map(function(effect_name) {
    # Select what we want, rename the column to effect to avoid the
    # messy NSE stuff. What does !!effect_name do? Expands the vector of
    # variable names into arguments for select. So we're selecting choice,
    # person_id, everything in effect_name, and group if the data is grouped
    #print(effect_name)
    if(grouped_data){data = conjoint %>% select(choice, person_id, !!effect_name, group) %>%
      # Rename whatever the variable's actual name is to effect so we can proceed more easily
      rename_at(3, ~ "effect")}
    if(! grouped_data){data = conjoint %>% select(choice, person_id, !!effect_name) %>%
      # Rename whatever the variable's actual name is to effect so we can proceed more easily
      rename_at(3, ~ "effect")}
    
    ### Print error if effect is empty
    if(! F %in% is.na(data$effect)){
      #print(paste("Data is empty for ",effect_name,"!",sep=""))
      return(data.frame())}
    
    ### calculate weighted mean support for the policy among group 0
    ### (if the policy is available)
    tmp <- distinct(conjoint,person_id,.keep_all=T)
    pol <- policies[which(positions_agreement==effect_name)]
    if(pol %in% names(tmp)){
      if(grouped_data){margin_yes <- weighted.mean(case_when(tmp[[pol]]==2~0,tmp[[pol]]==1~1,tmp[[pol]]==3~0,
                                                             tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp$group==0 & tmp[[pol]]!=999],
                                                   w=tmp$w[tmp$group==0 & tmp[[pol]]!=999],na.rm=T)}
      if(! grouped_data){margin_yes <- weighted.mean(case_when(tmp[[pol]]==2~0,tmp[[pol]]==1~1,tmp[[pol]]==3~0,
                                                               tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp[[pol]]!=999],
                                                     w=tmp$w[tmp[[pol]]!=999],na.rm=T)}
      if(grouped_data){margin_no <- weighted.mean(case_when(tmp[[pol]]==2~1,tmp[[pol]]==1~0,tmp[[pol]]==3~0,
                                                            tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp$group==0 & tmp[[pol]]!=999],
                                                  w=tmp$w[tmp$group==0 & tmp[[pol]]!=999],na.rm=T)}
      if(! grouped_data){margin_no <- weighted.mean(case_when(tmp[[pol]]==2~1,tmp[[pol]]==1~0,tmp[[pol]]==3~0,
                                                              tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp[[pol]]!=999],
                                                    w=tmp$w[tmp[[pol]]!=999],na.rm=T)}
    }
    if(! pol %in% names(tmp)){ margin_yes <- NA 
    margin_no <- NA}
    
    # condition on respondents in favor of the policy
    w <- weights
    if(use=="for"){
      pol <- policies[which(positions_agreement==effect_name)]
      data <- data[which(conjoint[[pol]]==1),]
      w <- w[which(conjoint[[pol]]==1)]
    }
    # condition on respondents against the policy
    if(use=="against"){
      pol <- policies[which(positions_agreement==effect_name)]
      data <- data[which(conjoint[[pol]]==2),]
      w <- w[which(conjoint[[pol]]==2)]
    }
    # condition on respondents DK or didn't answer on the the policy
    if(use=="DK"){
      pol <- policies[which(positions_agreement==effect_name)]
      data <- data[which(is.na(conjoint[[pol]]) | conjoint[[pol]]==3),]
      w <- w[which(is.na(conjoint[[pol]]))]
    }
    # condition on respondents taking a particular stance on each policy
    if(use=="direction"){
      pol <- policies[which(positions_agreement==effect_name)]
      dir <- direction[which(positions_agreement==effect_name)]
      data <- data[which(conjoint[[pol]]==dir),]
      w <- w[which(conjoint[[pol]]==dir)]
    }
    
    if(use %in% c("overall","all")){
      data$effect <- abs(data$effect - 2)
    }
    
    # FELM: Formula: regression (outcome on treatment). Second group is
    # fixed effects. Third group is instrumental variables. Fourth group
    # is what to cluster on.
    group_error=F
    if(grouped_data){if(0 %in% rowSums(table(data$group,data$effect))){
      print(paste("Group perfectly predicts treatment assignment for:",effect_name))
      group_error=T
      grouped_data=F
    }}
    if(! grouped_data){temp_cl = felm(choice ~ effect | 0 | 0 | person_id, data = data,weights=w)}
    if(grouped_data){
      # these two regressions are basically the same:
      data$group <- as.character(data$group)
      # first regression gets us significant differences from the effect in group 0
      temp_cl = felm(choice ~ effect + effect*group | 0 | 0 | person_id, data = data,weights=w)
      # second regression gets us main effects with correct SEs for each group
      temp_cl2 = felm(choice ~ effect:group + group | 0 | 0 | person_id, data = data,weights=w)
    }
    
    t_value <- 1.96
    if(bonferroni){t_value <- qnorm(1-.025/length(positions_agreement))}
    
    # Return the results with a CI and significance
    output <- tibble(
      condition = str_remove(effect_name,"agreement_"),
      effect = temp_cl$coefficients[2],
      se = temp_cl$cse[2],
      ci_low = effect - t_value * se,
      ci_high = effect + t_value*se,
      is_sig =
        case_when(
          (ci_high < 0) | (ci_low > 0) ~ 1,
          (ci_high > 0) & (ci_low < 0) ~ 0
        ),
      ci_int_low = ifelse(grouped_data,
                          temp_cl$coefficients[4] - t_value*temp_cl$cse[4],NA),
      ci_int_high = ifelse(grouped_data,
                           temp_cl$coefficients[4] + t_value*temp_cl$cse[4],NA),
      is_sig_int =
        case_when(
          (ci_int_high < 0) | (ci_int_low > 0) ~ 1,
          (ci_int_high > 0) & (ci_int_low < 0) ~ 0
        ),
      N_obs = ifelse(grouped_data,
                     length(which(c(data$group==0)[-temp_cl$na.action])),
                     temp_cl$N),
      N_people= ifelse(grouped_data,
                       length(unique(data$person_id[data$group==0 & ! 1:dim(data)[1] %in% temp_cl$na.action])),
                       length(unique(data$person_id[! 1:dim(data)[1] %in% temp_cl$na.action]))),
      margin_yes=margin_yes,
      margin_no=margin_no,
      group=0
    )
    if(group_error){output$effect = NA}
    
    if(grouped_data){
      fulloutput <- output
      ngroups <- length(unique(data$group[! is.na(data$group)]))
      for(i in 2:ngroups){
        tmpoutput <- tibble(
          condition = str_remove(effect_name,"agreement_"),
          effect = temp_cl2$coefficients[ngroups + i],
          se = temp_cl2$cse[ngroups + i],
          ci_low = effect - t_value * se,
          ci_high = effect + t_value*se,
          is_sig =
            case_when(
              (ci_high < 0) | (ci_low > 0) ~ 1,
              (ci_high > 0) & (ci_low < 0) ~ 0
            ),
          ci_int_low = temp_cl$coefficients[ngroups + i] - t_value*temp_cl$cse[ngroups + i],
          ci_int_high = temp_cl$coefficients[ngroups + i] + t_value*temp_cl$cse[ngroups + i],
          is_sig_int =
            case_when(
              (ci_int_high < 0) | (ci_int_low > 0) ~ 1,
              (ci_int_high > 0) & (ci_int_low < 0) ~ 0
            ),
          N_obs = length(which(c(data$group==i-1)[-temp_cl$na.action])),
          N_people= length(unique(data$person_id[data$group==i-1 & ! 1:dim(data)[1] %in% temp_cl$na.action])),
          margin_yes = ifelse(pol %in% names(tmp),weighted.mean(case_when(tmp[[pol]]==2~0,tmp[[pol]]==1~1,tmp[[pol]]==3~0,
                                                                          tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp$group==(i-1) & tmp[[pol]]!=999],
                                                                w=tmp$w[tmp$group==(i-1) & tmp[[pol]]!=999],na.rm=T), NA),
          #            ifelse(pol %in% names(tmp),
          #                        weighted.mean(case_when(tmp[[pol]]==2~0,tmp[[pol]]==1~1,is.na(tmp[[pol]])~0)[tmp$group==(i-1)],
          #                                      w=tmp$w[tmp$group==(i-1)],na.rm=T), NA),
          margin_no = ifelse(pol %in% names(tmp),weighted.mean(case_when(tmp[[pol]]==2~1,tmp[[pol]]==1~0,tmp[[pol]]==3~0,
                                                                         tmp[[pol]]==999~NA_real_,is.na(tmp[[pol]])~0)[tmp$group==(i-1) & tmp[[pol]]!=999],
                                                               w=tmp$w[tmp$group==(i-1) & tmp[[pol]]!=999],na.rm=T),NA),
          #  ifelse(pol %in% names(tmp),
          #                            weighted.mean(case_when(tmp[[pol]]==2~1,tmp[[pol]]==1~0,is.na(tmp[[pol]])~0)[tmp$group==(i-1)],
          #                                          w=tmp$w[tmp$group==(i-1)],na.rm=T), NA),
          group=i-1
        )
        fulloutput <- bind_rows(fulloutput,tmpoutput)
      }
      output <- fulloutput
    }
    
    if(use=="direction"){
      dir <- direction[which(positions_agreement==effect_name)]
      # reverse the margin when the direction is reversed
      #if(dir==2){output$margin <- 1-output$margin}
    }
    if(use=="against"){
      # reverse the margin
      #output$margin <- 1-output$margin
    }
    
    output
  }) %>%
    # bind_rows will combine across all the coefficients.
    bind_rows() %>%
    # Sort data internally
    arrange(-effect) %>%
    # Reorder by effect size descending as far as ggplot2 is concerned
    mutate(condition = fct_reorder(condition, effect))
  
  # match on descriptive labels, if available
  if(! is.null(lab)){
    result$condition <- as.character(result$condition)
    ind <- match(result$condition,lab$label)
    result$label <- lab$newlabel[ind]
    # if the direction is provided, add this information to the labels as well
    if(use=="direction"){
      ind <- match(result$condition,which_effects_conjoint)
      result$label <- paste(c("","(not) ")[direction[ind]],result$label,sep="")
    }
    result$condition <- factor(result$condition)
  }
  
  # Return table 
  result
}